#!/opt/lampp/bin/perl -w
use strict;
use warnings;
use DBI;

require('function.pl');
my $db_host = 'localhost';
my $db = 'crm';
my $db_user = 'zjxia';
my $db_password = 'bblovebb';
my $db_conn = DBI->connect("DBI:mysql:database=$db;host=$db_host", $db_user, $db_password);
$db_conn->do('set names gbk');
my %keywords = load_map('keyword_map_baixing');
my %cities = load_map('city_map_baixing');
load_proxies();
my $counter = 0;
while (my ($city_cn, $city_en) = each %cities) {
	while (my ($keyword_cn, $keyword_en) = each %keywords) {
		my $list_url = "http://$city_en.baixing.com/$keyword_en/";
#		my $list_content = `curl -s $list_url`;
		my $list_content = fetch_url_via_proxy($list_url);
#		print "$keyword_en\n";
		my @info_urls = parse_list($list_content);
		foreach my $info_url (@info_urls) {
			if (++$counter % 100 == 0) {
				check_spider_block();
			}
			if (index($info_url, 'http://') < 0) {
				$info_url = "http://$city_en.baixing.com$info_url";
			}
			my $info_content = `curl -s '$info_url'`;
#			my $info_content = fetch_url_via_proxy($info_url);
#			print "$info_content\n";
#			print "http://localhost/meta_search/58lib/58_info.php?$info_url\n";
			my ($title, $url, $phone_num, $contact_person, $location, $crawl_number) = ('', '', '', '', '', int(rand(10)));
			if ($info_content =~ /var title = '([\d\D]+?)'/) {
				$title = $1;
			}
			$url = $info_url;
			my $phone_num_content = `curl -s 'http://localhost/meta_search/baixinglib/baixing_info.php?textcode=$info_url'`;
			if (index($phone_num_content, 'wy') == 0) {
				$phone_num = substr($phone_num_content, 2);
			}
			elsif ($info_content =~ /<span id="mobile_area">联系方式：([\d\D]+?)<\/span>/) {
				my $phone = $1;
				if ($phone =~ /([\d\-]+)/) {
					$phone_num = $1 if (length($1) > 6);
				}
			}
			if (0 && $info_content =~ /<small>([\d\D]+?) 的联系方式:<\/small>/) {
				$contact_person = $1;
			}
#			print $info_content;
			if ($phone_num eq '') {
				print "no phone num $info_url\n";
				next;
			}
			if (execute_scalar("select count(*) from snapshot where phone_num = '$phone_num'", $db_conn) > 0) {
				print "duplicate $phone_num\n";
				next;
			}
#			print "insert into snapshot(domain, query, city, title, url, phone_num, contact_person, crawl_number) values('58'
#                        , '".utf8_to_gb2312($keyword_cn)."'
#                        , '".utf8_to_gb2312($city_cn)."'
#                        , '".utf8_to_gb2312($title)."'
#                        , '$url'
#                        , '$phone_num'
#                        , '".utf8_to_gb2312($contact_person)."'
#                        , $crawl_number
#                        )";
			$db_conn->do("insert into snapshot(domain, query, city, title, url, phone_num, contact_person, crawl_number) values('baixing'
			, '".utf8_to_gb2312($keyword_cn)."'
			, '".utf8_to_gb2312($city_cn)."'
			, '".utf8_to_gb2312($title)."'
			, '$url'
			, '$phone_num'
			, '".utf8_to_gb2312($contact_person)."'
			, $crawl_number
			)");
			print "$keyword_cn\t$city_cn\t$title\t$url\t$phone_num\t$contact_person\n";
		}
	}
}

sub parse_list {
	my @urls;
	while ($_[0] =~ /<a href="([^\s]+?)" class="topicLink" target="_blank">/g) {
		push @urls, $1;
#		print "\t$1\n";
	}
	return @urls;
}

sub check_spider_block {
	my $counter = 600;
	while (1) {
		my $content = `curl -s http://beijing.baixing.com/`;
		if (length($content) < 100) {
			print "sleep for $counter\n";
			sleep($counter);
			$counter *= 2;
		}
	}
}
